#!/usr/bin/env python
# encoding: utf-8

import imp
import sys
from bookspider import BookSpider
from lxml import etree

'''
www.xbiquge.so 网站爬虫
抓取，生成电子书原材料
'''

class xbiquge(BookSpider):
  def __init__(self, book_id):
    super().__init__()
    self.page_base = 'https://www.xbiquge.so/book/'
    self.book_id = book_id
    self.name = ''
    self.page_list = []

  def prepare(self):
    index_page = '%s%d/' % (self.page_base, self.book_id)
    rsp = self.download_url(index_page)
    # get book name
    doc = etree.HTML(rsp.content)
    self.name = ''.join(doc.xpath('//h1/text()'))
    # get book url range
    # 2022.03.19 网站将最新章节放在最前面了 所以要去掉
    # items = doc.xpath("//div[@id='list']//a/@href")
    items = doc.xpath("//div[@id='list']//dt[position()>1]/following-sibling::dd/a/@href")
    # return page list.
    self.page_list = ['%s%d/%s' % (self.page_base, self.book_id, item.title().lower()) for item in items]

  def book_name(self):
    return self.name or '笔趣阁书籍'

  def Referer(self):
    return '%s%d.html' %  (self.page_base, self.book_id)

  def sections(self):
    return self.page_list

  def process_page(self, pos, url, page):
    doc = etree.HTML(page)
    title = ''.join([item.text for item in doc.xpath('//title')])
    t_pos = title.find('_')
    title = title[:t_pos]
    body_els = doc.xpath("//div[@id='content']")
    if not body_els:
      print('%d [%s] 解析body失败' % (pos, url))
      return None
    # add title to body elements.
    title_el = etree.XML("<h1>%s</h1>" % title)
    body_el = body_els[0]
    body_el.insert(0, title_el)
    # 正文最前加了网站广告，要过滤掉
    content = body_els[0]
    content.text = ''
    body = etree.tostring(body_els[0], pretty_print=True, method='html')
    return {
      'title': title,
      'body': body,
    }


def main():
  xb = xbiquge(int(sys.argv[1]))
  # xb.crawl_from(168)
  xb.crawl()
  xb.make_epub()


if __name__ == '__main__':
  main()
